{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from collections import deque\n",
    "from matplotlib import pyplot as plt\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "import torch.nn.functional as F\n",
    "from torch.distributions import Categorical"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Network(nn.Module):\n",
    "    def __init__(self, state_space, action_space, seed):\n",
    "        super(Network, self).__init__()\n",
    "\n",
    "        self.model = nn.Sequential(\n",
    "            nn.Linear(state_space, 16),\n",
    "            nn.ReLU(),\n",
    "#             nn.Linear(64, 64),\n",
    "#             nn.ReLU(),\n",
    "            nn.Linear(16, action_space)\n",
    "        )\n",
    "\n",
    "    def forward(self, x):\n",
    "        return F.softmax(self.model(torch.tensor(x).float()))\n",
    "    \n",
    "    \n",
    "    def act(self, x):\n",
    "        x = torch.from_numpy(x).float().to(device)\n",
    "        # https://pytorch.org/docs/stable/distributions.html#score-function\n",
    "        probabilities = self.forward(x)\n",
    "        m = Categorical(probabilities)\n",
    "        action = m.sample()\n",
    "        log = m.log_prob(action)\n",
    "        \n",
    "        return action, log"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import gym\n",
    "env = gym.make(\"CartPole-v1\")\n",
    "\n",
    "state_space = env.observation_space.shape[0]\n",
    "action_space =env.action_space.n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "max_episodes = 2000\n",
    "max_t = 500\n",
    "update_every = 4\n",
    "gamma = 0.99\n",
    "win_condition = 200"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def update_model():\n",
    "    discounts = [gamma ** i for i in range(len(reward_log)+1)]    \n",
    "    rewards = [discount * reward for (discount, reward) in zip(discounts, reward_log)]\n",
    "    G = np.array(rewards).sum()\n",
    "    \n",
    "    policy_loss = np.array([ -log * G for log in policy_log]).sum()\n",
    "    \n",
    "    optimiser.zero_grad()\n",
    "    policy_loss.backward()\n",
    "    optimiser.step()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ubuntu/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:14: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  \n",
      "/home/ubuntu/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:14: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " Episode: 0.0\tAverage Score 17.00\tReward: 17.00\n",
      " Episode: 100.0\tAverage Score 28.02\tReward: 37.15\n",
      " Episode: 200.0\tAverage Score 37.74\tReward: 47.84\n",
      " Episode: 300.0\tAverage Score 41.28\tReward: 57.15\n",
      " Episode: 400.0\tAverage Score 46.26\tReward: 58.41\n",
      " Episode: 500.0\tAverage Score 50.47\tReward: 70.91\n",
      " Episode: 600.0\tAverage Score 54.68\tReward: 97.73\n",
      " Episode: 700.0\tAverage Score 69.59\tReward: 196.80\n",
      " Episode: 718.0\tAverage Score 72.77\tReward: 200.11Environment Solved!\n"
     ]
    }
   ],
   "source": [
    "model = Network(state_space, action_space, 1234).to(device)\n",
    "optimiser = optim.Adam(model.parameters(),lr=1e-2)\n",
    "\n",
    "scores_deque = deque(maxlen=100)\n",
    "\n",
    "score_log = []\n",
    "average_score_log = []\n",
    "\n",
    "for episode in range(max_episodes):\n",
    "    state = env.reset()\n",
    "    \n",
    "    policy_log = []\n",
    "    reward_log = []\n",
    "    \n",
    "    for t in range(max_t):\n",
    "        action, log = model.act(state)\n",
    "        state, reward, done, info = env.step(action.cpu().item())\n",
    "        \n",
    "        reward_log.append(reward)\n",
    "        policy_log.append(log)\n",
    "        \n",
    "        if done:\n",
    "            break\n",
    "        \n",
    "    update_model()\n",
    "    \n",
    "    scores_deque.append(np.array(reward_log).sum())\n",
    "    \n",
    "    # Book Keeping\n",
    "    score_log.append(np.array(reward_log).sum())\n",
    "    average_score_log.append(np.array(scores_deque).mean())\n",
    "    \n",
    "        \n",
    "    print(\"\\r Episode: {:.1f}\\tAverage Score {:.2f}\\tReward: {:.2f}\".format(episode, np.mean(average_score_log), np.array(scores_deque).mean()), end=\"\")\n",
    "    if (episode % 100 == 0):\n",
    "        print(\"\\r Episode: {:.1f}\\tAverage Score {:.2f}\\tReward: {:.2f}\".format(episode, np.mean(average_score_log), np.array(scores_deque).mean()))\n",
    "\n",
    "    if np.mean(scores_deque) > win_condition:\n",
    "        print(\"\\rEnvironment Solved!\")\n",
    "        break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x7fa3a4dd9048>]"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD8CAYAAAB5Pm/hAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzsnXmYHFX1v9/by+yZ7MlknywkIQskEJJAgIQ9LLKJbLL5QwFBBVERFVTABfErKigCEhUEBJWdsIWQgIHsZAWy7/s2+9rddX9/dFVPVXf1NtPTM5Oc93nmmepbt6puV1d96tS5556rtNYIgiAIhy+etm6AIAiC0LqI0AuCIBzmiNALgiAc5ojQC4IgHOaI0AuCIBzmiNALgiAc5ojQC4IgHOaI0AuCIBzmiNALgiAc5vjaugEAPXr00KWlpW3dDEEQhA7F0qVLD2iteyar1y6EvrS0lCVLlrR1MwRBEDoUSqmtqdQT140gCMJhjgi9IAjCYY4IvSAIwmGOCL0gCMJhjgi9IAjCYU5KQq+U2qKUWqWUWq6UWmKWdVNKzVJKrTf/dzXLlVLqEaXUBqXUSqXUca35BQRBEITEpGPRn6a1Hqe1nmB+vhuYrbU+CphtfgY4FzjK/LsJ+EumGisIgiCkT0vi6C8CppnLTwNzgR+a5c/o8ByFC5RSXZRSfbTWu1vSUEEQhNaisj7AP+dvpbIuQK7PwxlH92broVqmjehJcZ6/VY65u6KOz3dVcsbRvVtl/3ZSFXoNvKeU0sATWusngd428d4DWK3tB2y3bbvDLHMIvVLqJsIWPwMHDmxe6wVBEDLAh2v389t310Y+v7ZiF1sP1jJ9dAmPX3t8qxzzkj9/wp7KerY8eH6r7N9OqkJ/stZ6p1KqFzBLKbXGvlJrrc2HQMqYD4snASZMmCAzlAuC0GY0BA3H56r6IAC7K+tb7Zh7WnHf0aTko9da7zT/7wNeASYCe5VSfQDM//vM6juBAbbN+5tlgiAI7ZJgyEheqQOTVOiVUoVKqU7WMnA2sBp4HbjerHY98Jq5/DpwnRl9MxmoEP+8IAjtlQPVDdz98ipHWbiL8fAhFddNb+AVpZRV/3mt9TtKqcXAv5VSNwJbgcvN+m8B5wEbgFrgaxlvtSAIQob4YM2+mLLDS+ZTEHqt9SbgWJfyg8AZLuUauC0jrRMEQWhl/F7VpsfXWmMa0q2GjIwVBOGIxueJlcFsem6ycSwRekEQjmh8nja26LNwDBF6QRCOaLwuQp/NzthsHEuEXhCEIxqfi48+m52xYtELgiC0Mm4++mwiPnpBEIRWxs2iz6ZJr7NwMBF6QRCOaLwuoY1Zdd2IRS8IgpB9DreRsSL0giAc0bhJulWWjcBLsegFQRBambY23sVHLwiC0Mq4Ca2MjBUEQThMMAzNDX9fHFOeDSu76Vitjwi9IAhHLIdqG2kMxuaiz65FL64bQRCErBPpjM1Cb2w2niktmRxcEAShw3LzP5fw2a5K95WHmY9ehF4QhCOSdz/b29ZNCCOdsYIgCNknu52x4qMXBEHIOhJeKQiCcBhyxYQBbXJcCa8UBEHIEh7bBCTZTWomrhtBEISs4G0jNRSLXhAEIUt4bEHz2Z1KsPWPIUIvCIJAlNBn8bgSdSMIgpAl7KNgs5rRUix6QRCE7ODJRr4DF8RHLwiCkCW8njYSerHoBUEQskMbGfTioxcEQcgWbea6EYteEAQhO7SR50Z89IIgCNnC22YWvbhuBEEQsoIS1w0opbxKqWVKqTfNz4OVUguVUhuUUi8qpXLM8lzz8wZzfWnrNF0QBCFztJWPPhukY9HfDnxh+/wb4Pda62FAGXCjWX4jUGaW/96sJwiC0K5pMx99e7HolVL9gfOBp8zPCjgd+K9Z5WngYnP5IvMz5vozVFu9EwmCIKSIp63i6NtReOUfgLsAa7r07kC51jpoft4B9DOX+wHbAcz1FWZ9B0qpm5RSS5RSS/bv39/M5guCIGSGIzq8Uil1AbBPa700kwfWWj+ptZ6gtZ7Qs2fPTO5aEAQhbQ7n8MpUJgefAlyolDoPyAOKgT8CXZRSPtNq7w/sNOvvBAYAO5RSPqAzcDDjLRcEQQBW7aggYBgcN7Bri/bTdhZ9O3DdaK1/pLXur7UuBa4EPtBafxWYA1xmVrseeM1cft38jLn+A53N5M6CIBxRfOlP87j0sU9avJ+289G3Pi2Jo/8hcKdSagNhH/wMs3wG0N0svxO4u2VNFARBaH28LjqfDenPhhmciusmgtZ6LjDXXN4ETHSpUw98JQNtEwRByBptZdFnw6aXkbGCIAjIyFhBEITDCrduw8M56kaEXhCEIw7DRV2P6Dh6QRCEw42Qi9K3WfZK8dELgiBkHsPFjG6zGabEohcEQcg8bkIvrhtBEITDCFcffRupobhuBEEQWgE3H71Y9IIgCIcR7uGVh282dRF6QRCOOMSiFwRBOMwJtasBU+KjFwRByDhuVrSkQBAEQTiMcB0wJWmKBUEQDh/c4+jboCG0k4lHBEEQDjcMI7aszTpjs3AMEXpBEI44sp0Coa4xRFlNo+s68dELgiC0Am5RN63po7/ksY8Z/8CsOGvFdSMIgpBxsj1gas2eqgRtabXDRhChFwThiCMkPnpBEITDG8tHf+dZwyNlbRd10/rHEKEXBOGIw4qjH1HSKVLWVpODS3ilIAhCK2Bpq31WqUy5buoDIR6dvZ7GoIt/yK0tGTlqYkToBUE44rCibuw56N10vjlpEZ74cBO/m7WO5xduTam+uG4EQRBaActHb7fi3SS9OTZ+bWMQgPqULXpx3QiCIGQcw3AR+jabNLb1D+Fr/UMIgiC0D+oDIQytI52x9kFSbTXtiPjoBUEQMshJD37AqJ++G5kz1m7EZ8qgX7TlUFr1xUcvCIKQQQ6Z+WYsH73X4aNvudJv2FfNsm3l5v5SQ3z0giAIrUBE6O2umwxY9OW17onLEiEWvSAIQitg+eiz3QEbDBnUNAQdZeKjFwRBaAUiA6YybNEn455XVzP6Z+86ZrhqFyNjlVJ5SqlFSqkVSqnPlFL3meWDlVILlVIblFIvKqVyzPJc8/MGc31p634FQRCE9AhFwiubyjLho7djPTgMm6i/sHi74/jQfiz6BuB0rfWxwDhgulJqMvAb4Pda62FAGXCjWf9GoMws/71ZTxAEod3gOmAqAzrvto/fvLMmpszRAdsefPQ6TLX50W/+aeB04L9m+dPAxebyReZnzPVnqDYbiSAIghBLawm9mxfmpU93JKzXbqJulFJepdRyYB8wC9gIlGutrV6FHUA/c7kfsB3AXF8BdM9kowVBEFqC4eajb6UhU8lc8O0m6kZrHdJajwP6AxOBkS09sFLqJqXUEqXUkv3797d0d4IgCCnj6qPPsOsm0YPDPmdtuxF6C611OTAHOBHoopSyUij0B3aayzuBAQDm+s7AQZd9Pam1nqC1ntCzZ89mNl8QBCF9Iq4bm9K3Vjp6Nx3X2XXRpxR101Mp1cVczgfOAr4gLPiXmdWuB14zl183P2Ou/0BnI35IEAQhRdx89NnMduO06FtfHlNJatYHeFop5SX8YPi31vpNpdTnwAtKqV8Ay4AZZv0ZwD+VUhuAQ8CVrdBuQRCEZmPNGevNcGesnUT703GWW4ukQq+1XgmMdynfRNhfH11eD3wlI60TBEFoBSyL2ulTbx3cLHZt2Ne30oFtyMhYQRCOOAy3NMWtFAXu6qPPsk0vQi8IwhGHW3hlZkhtf47OWLHoBUEQMk/IxXWTGVzcNC5C7uiMzXQTXBChFwThiEO75KPP6vEdbWn944nQC4LQYWluaGLIZc7YzJDa/pwWvfjoBUEQ4tJca9jy0Xtaa5SUDdeHkfjoBUEQUqO5GmnYUiBcO3kQvYtzXeu1xOBPFMVj2IW++YdIGRF6QRA6LEYzzWH7VIIPXDyGhT8+M5PNcpAsvLJdTDwiCILQ3pizZh+vLNvRbLdHyDUFQvYwsmHG20glBYIgCEK74mv/WAzAuWP6xK2zckc5uT4vI0o6xawzWq0zNjV0lrNXitALgnBYcuGfPgZgy4Pnx6wLuqQpzgxNqq1ii5pqtceJRwRBENojzfXRB0OxKRAyQaouGRkZKwiCkCLNFcmAEc4qlun8Nqm2x9kZm9EmuCJCLwhCh6UlFn0q1ny6u3fNVOlST8IrBUEQUqS5Ijlj3uaU/PMa+Mkrqyi9e2ba7UmYjz7LE4+I0AuC0GGx53VPl0DIKbCu1rjWPLdwW+rtce14jS0Ui14QBCGK5xduY9WOiphyu697yZZDGT9uuiKcquvGmQMhzYM0AxF6QRDaPT9+ZRVf+tO8mHK7ZXzZ4/NbdAw3n33aPvpU60l4pSAIQmpk0r89sFsBPzhnBOePbRqElb5F37SsXMosHK4bsegFQRDik8lUAkopbjttGP265jcVptlpmqp1LhOPCIIgpEhruz3sew+l8FSxV0kUox8INfUii0UvCIKQgNYQSbs82/cfSsWit9WxdN7tYdQQtAm9+OgFQRDiE097n/rfpubv1Kb0dhE2UgjlTFWyGwLZteglqZkgCB2W6JGxf56zgX2V9Tw9f2tG9p+KRV9W08jG/dVMKO2WstI3BEOR5cJcb0uamBIi9IIgtGsSdYJGr/ntu2tT3u+t04a6liubSe8Q+pB7O6766wLW7Kliy4Pnu7ph3JpvuW4euuwYLhnfP+U2Nxdx3QiC0K6J7gM1DO26nA7XnTiIu6aPdF2nHK6bJuJZ9Gv2VIXras33/r2iaT8Jjm9Z9Lm+7EiwCL0gCO2aaPdMIBVneRJSnXDE/jaRLOrG0FBWG0hpv5aP3u8VoRcEQYgR2KDNhdLc7JW+BBnN4q1Jdqx4DwK3Ust1kyNCLwiCEOvjtgt9cyNWvN7ULPraxqZO02ASiz6YxpuG5brxi+tGEAQhseumuZGJCS1626qy2samdiQRenvIpAO3ztiAWPSCIAgRQlFpCOwWfSqjVd3weuJLnz3qpqo+mPRY1oOhMeQU+kQti7hufNmZnDyp0CulBiil5iilPldKfaaUut0s76aUmqWUWm/+72qWK6XUI0qpDUqplUqp41r7SwiCkHm01nyy4UBWJsZI2A6bfhra6SK5+6WVzdpnIos+HvGibqw9NQajhN6s7j4yNuy6yfG2fgw9pGbRB4Hvaa1HAZOB25RSo4C7gdla66OA2eZngHOBo8y/m4C/ZLzVgiC0Ov9atJ2rn1rIGyt3t2k77AJraO0Yobpka1mz9ploGsF4ATnxLfrwBg0xQq/N/7HbWHX97cWi11rv1lp/ai5XAV8A/YCLgKfNak8DF5vLFwHP6DALgC5KqT4IgtCh2HqoBoCdZXUJ6/36rS/4aN3+VmuHESX0qeScSUZzom7iCr353z7aFZK4bkwffX7dnqzkQEjLR6+UKgXGAwuB3lpr61G/B+htLvcDtts222GWCYLQkUhRf574aBPX/W1RqzXDkdJXQ6iZcfSb9ldHlicN6Z7ydl/xzuXVnHsZ+Npl8OFDMP/PUNs0m5X1BhC20jUlHORh/2P0PfCJbS+aoWonHsJtDxgG49QGBjw/DZbMaNb3SYeUUyAopYqAl4A7tNaV9hScWmutlErrsaSUuomwa4eBAwems6kgCFkkxbFFrYZd1+sDIZZsaZ675vTffQjA16aUMm5Al/gVzS/sJ8htvle5w/cyXxgD8VfvhDm/DNd5/+dw/A1ghHjB+z/83gClM3vyRe5y8pUZqbNsHmx7mKd8RUzzLAdgpTGY90PHE6o9h9N9b6G9OTDivGZ9n3RISeiVUn7CIv+c1vpls3ivUqqP1nq36ZrZZ5bvBAbYNu9vljnQWj8JPAkwYcKEtu3tEQSh3WK36O9/43NeXhYjJ2mRWkij5jf+J7nUO4/3QsfzrcB3ePErpzK+h4YD6+C9n8CiJwE43gM7dXc8jXnkq0bKdSE/CNzM/xvt4cSKtyIiD3CMZzPHeDbDlv+CF2qGXEVhcd8WfZ9USCr0Kmy6zwC+0Fo/bFv1OnA98KD5/zVb+beUUi8Ak4AKm4tHEIQOQnuxvuy+8dW7YicIT5skbygKGKG2c6l3Hn8OXshvg1cAipAGCrtD4YnwjQ8gFASPl9PvfYZdwU786awpfP2ZJSgMNB4mDTyaEyf/gO/ddx8fBUcDcJNvJv8OTeO+nnPoWb6S4om3U9jyb5SUVCz6KcC1wCqllPVo+jFhgf+3UupGYCtwubnuLeA8YANQC3wtoy0WBCGrJNLFbIReZvoQyfLc5AUr+Io37Ob5V+gMrDMQ0xnrDcvnLtWLeoxIJI02uz61Bvx5vBI6Gcv79MvgNQA80/MHvLNvD/O7lrb4+6RCUqHXWs8j/m99hkt9DdzWwnYJgtDGpCLidvHbdrCWvBwPvTrlZbQd0Z2xrc0ln91Oie8zVhhD2KF7RMrjx9GH5bExFB11E7+x1r5STa7WUiQfvSAIzcae/+XU384BYMuD52f0GNFx9C0l4T5qD1FS/RkbjT58s/EO7DZuvGAfK1Iz/oAplzaY5y1b/dySAkEQhIQkMjozIbzJyLR7KOHuti0A4EeBr7OLHo5V8ZKWxR0wleAw1sMr0QTimUSEXhAEV1LR1+bmmoGwVWu3glfuKGfR5kMx9UItTD8fDBn8c/6WyOeED46tHxNUOazQsbNPxXuoJUuB4IZ13rIVuipCLwhCQlQCB0NLhP6ul1Yy/J63I58v/NPHXP7E/Jh6Dh99M47z7IKt3PvaZ7b9Jai8ZR57isfQQE7MqrgPHMeAqSY0mjdW7HIVfCPLPnoRekEQmk1LhP6/S3ekVK+l7qHoWZ/itrl8O+xZya7OxwNNvvdk2zWlQIi16L/9r2Wu21j7akZutWYhQi8IgiupyGtLhD5VWjpzYLSrJq7rZvFfQXn5ouRLAPiiUhknS2oW7bpJhPWdEr0tZRIRekEQEpLIu5CJBGPJaKlFH63Pcfe2ZR70n0B1fjg1V3SGy7jhlRHXTVR4ZYJ2Rzpjs6TAIvSC0E7YX9XA3sr6tm5GWtgnAWkuyWZucghsMw4XHc/u+uBoqIZdy2HQlEhRdIbLeO1sUWds/CoZRYReENoJJ/zyfSb9anZbNyNCKoZ0q8e147SMm3O06IeRq15vXwg6BINOihR5oi36NF03idoqnbGCIHQYkk2YnQrJdtHSQzzx0SbHZ1eXyvr3wJcHAydHXDHRFn3I0FTWB9h+qDZS9uqynRyqCWerDERPJSjhlYIgAGw9WMPxD8xyiEdHIpnbZfXOClZsL0+8jyQWvd2SzsTgKdfO3fXvweCpkNOUYszNR3/+I//jlIfmRMrueLEpM2Ug6s0hYQoEQyx6QThi+M+SHRysaeTlT1uWeretSGbRX/DoPC7688cJ69z76mpqG4Nx12d69G3M/sq3w6FNMPQ0oCkSxs2i334o/mxb6Vj02RhRbEdy3QhCG5KfE54cuj4qYqOjkInwyv8s3UFpj/jJepdta3ojyIQ8xjR5czhTJYOnAk3uFK83qjM2iTjHCH2CumLRC8IRRK4vfAvWB9qf0CdyPVhkKo4+WiTt/PbdtU1tcjncsf07p3WsGPfP6pegqAR6He0o9qrUOmMtol03iUx6a1cyYEoQMsQf31/P9/+zoq2b4Uqe37To4wj9vsp6Tvz1bDba5jttT2Qjjt5O0OWBkG5iMIdlXr4NNn4AE78RMeWtvcX46JMKffoWvSQ1E4QM8fv316U83D7b5EeE3t2ifWvVbnZX1PP0J1uy2ConicQoUxZ9qs+LXRWx4wzStYodTd74Qfj/yAsiRRHXTQuFPlF9SYEgCEcQOUlcN1Ysd7Y776BJfB9483PeXLnLtU7mhL75+0nXz+04lyv/Dd2GQs8RSfeb7O0lumM6UUoEqw1i0QtChsnGtHfRlNU0UmbGWbth3fDxhN4SgjZouoNvPZ84OVdLacleUhX6ayYPDB/LOlj1ftj6CRxzuSOgPV7+mWShpNHC3pig3yFk6KzF0IMIvZBl9lc1MG/9gVbb/+qdFZz18IdU1Qdi1sVzj7Qm4x+YxfgHZsVd3yT0cSa1MP+3sc7HJRtJzZKSgmD275rPiUPCE4lEOpnXvwtoGHGu+26jlDiYJJ4/2nWTzKLPos6L0AvZ5fIn5nPNjIWttv/fvbeW9fuqWbwldgKLShfxzzbPLtjqGEBkaUOtzaK3i6cnYtG3A0F1Ids+ejdSEUylmvzhkTavfRuK+0HJMTF13bBb9G7fe+P+Gsfn6LTFdkKGzlpoJYjQC1lm84HwzdBawpVor5V1bS/097y62jGAyBIP+xuI3TK0tKCd6nzGhL4lfRCpCqZloRsaCNTDxjkw/Jy4yh5davfRpxJtNHftvrjrxHUjHBG09iu/m5+1PVj00ViCUWGbHMPuIoi4btqp0Gci1w200EefgoopVMSi1xrYMAsCNTAi/kTm0UJs98ykkiM/esITO4bOXkcsiNALbUSqAqG15u1Vu1N+MCQSxMr6+MPs2wrre5Xb3jYCQReLvg289Km8dbVFNFA0qVj0YdeN6QYzDFjwF+jUB4ZMc6mrItvYCdnUvaXjB0KG+OiFI4BEArGnoj7ivnh9xS6++dynzJi3KW59O257tUaftqXr5nv/XuEatWGdB/uDLGDYhd7mbmgH7K2sj3Qy1jQEOVDdkJH9JtLNzvl+xvaLP/o1FctY0WT5X3/wd7D1Y5h6F3hjs8CoyP/oOHr7cguFXouPXjgCiHejfPfF5Uz+9WzufmkVAAerw6GJu8rTnJDDdg8V5oZv5ra06F/6dIdDwC3czoN9KH17ct3UB0JM+tVs7n55JQDn/OEjfmqbdNsiWRiiGwlnYzI0fm98UUxl0JFSCqUUX/fO5NTqd2DSN+H4ryXZpmm5OM/nME6a8x3tGIbO2mApEKEX4vDPBVtZv7eKf87fQundMxNmF2wO8YT+lWXhLI5vr94NNN3EqboILMGw30OFueHRp9my6GsagvzuvbUx4XVuszG5nQf7MP+2jLqJPqL1lvXu6j0A7Chzz+TYUrfGkqiIqZCh8XnjS1Uyy9iDwaTgEo758Cbu8T/H6tzxcPYD8TthXYq9HuX4rVrsutE6qz56yV4puHLvq6vxexW9OuUBYcu6oFvmLhd3S7ZJ4CyR9DZzZKi9tjXJc7Y6Yx+ZvZ4nPtpEn875jvJPt5XF1HX7Xg6LPuKjdzJn7T5OHNI9kiunNYhumo76H4+wBZ7msWzLlz0+ny0Pnm+2QRMyNDk2ob9kfD9OHd6D774Yzl/keKjneKlpDIeqlnCQ7/he5sveeeTWB2g81IO3Qycwq8s3edjrT9om+369Ho+jX8kwdIsevlpn9+F9xFr09YEQ5bXxRyxmi2ueWsjZv/+wrZvhwLoAAyEd8WtmotNtjy1PiZtFZB9Bat1U6fioAyGD/5mDsezWc9B0mVTWZcd1U2O+/QSjXDXXzlgUU9dt8KR7eGXT91m9s4Kv/X0x97/5eQZaG5/o3+jh99YB6U0UkvKxXLZ5Y8Uuhv74LRpDhsN1E7auY9t589Qh3HjKEAC6UclzOb/iCu9cFhvD+XneXSy55H98M/Bddnv7JWxL5EjKfkynuyZo6Bb3m1gPpGxwxAr9BY/OY9z98UcsZot5Gw6wbm/7ykxov4CbXAct2+fiLYeY/Oum+VDdwtMOuqQKsI6fik/UcvuAM0LCWnQbLdsaWOcqlRdzN9EMOnz05vm3rS83w/aeX7itVa3C6H3/w0yspjUs2HQw7nbNcWu4ZaV8ddnOyLXot1n0fq9y/L4jS4oBOG1EL9AGk9QXfJR7B0M9u7k1cDvXBH7C//xTKOkWrnfC4G4J2xKJurGVeZVyxtEbun2MCk6RI9Z1s2Ff+xLX9oT9ArYu9pZa9Gv3VDk+R1u7gGtOmHR89LUNTRZ7wMWiz9aNaR0mFR+sW5saXSx6ezW74LyxcjcXHtu3eQ1NQrxYcQ3MmLc57nYhl76IZDw9f2tMWeeCJveKXeijLfofTh/BVyb0Z2hwI31W38GduZ9ySBdxacPP6T36VDD7FIb0LGLu96cxoFtBSm2y/3wej3IYG4buWEJ/xFr02WLLgRqunbGQmob2F8MdD7uoejIU3hc9LZubiFS5nKN0jm/3oTo6zozYstbFcjslr5msMzayR+0UGYu9Lml7M0U8y1xrzfDeRUm3a+nbhv2esbtufB5PxKL/6qSBqO2LGPrW1fDEqQysXMZDgSs4rfFhPtXDm96uzB+jtEdhTPrhaOy/26Xj+3HnWcPxeVws+jS/X2GOlz9eOS6tbTJFUqFXSv1NKbVPKbXaVtZNKTVLKbXe/N/VLFdKqUeUUhuUUiuVUse1ZuM7Ar95Zw3/W3+AD9ftp7oh2Kzolar6QMajXhLhsOjTjHqJh8dlouVEx23O8R2x6DaxDGXbojcPnUqctKvrxqWd9mpuor+jrDbjOevjnXOtEw94syzflp5ve5+Klc4ZnBEwXo+C178N+z6H0+/h90e/wGOhiwj6w3H31kCzdOJblO3/w1eM4ztnHIXHo5ydsVpzp21i8FTI8Xm4aFzi/oHWIhWL/h/A9Kiyu4HZWuujgNnmZ4BzgaPMv5uAv2SmmR0XKywsEDIY87N3mfSr2Um2iOWY+95j0i/T3665hFws+kRTvaVC7ETLqcWUR/cRrNtbxc7yOj7ecICGqHlW7Teim3WfrdmQLIFMJU46nuumMWjw0br9kX3ZR8baT521+VV/XcDPXv+MqvoAq3dWsK+q5ZZ+vNNlaO0aKmoRTHK+F246mNRw+fErq5hv6wewu27C1nV4eVDtajiwFk7+Lpz6A/Z6w26s/Bxfwu+QCnbXm1c5XTeNQc17n+9NaT93njU83JbmN6XFJBV6rfVHQHQqwIuAp83lp4GLbeXP6DALgC5KqT6ZamxH4fq/LeLyx+cDTQJn3RhVzRi0o3XYrZGtcCz7BW0JbUuts9jZemLruB0jOrzy7N9/xJQHP+CrTy3k12+tibu9q9BnyaJP5yhuYhgMaZ5fuJXr/raIN1eExxMYLlEm4WOFl7cfqovUu+DReZz7h/+l3/DotsU5Xxp391L0dm7uuX1V9Vzx5ALufDHLEMWZAAAgAElEQVTx1I7PL9zm+GwX+tNG9iJkGAxSe7h800+gy0AYdzXQZJBYYyea+ksSHs6JS+XoOPp0QnVPHd4zjYO3Ds3tjO2ttd5tLu8BepvL/YDttno7zLLdHEF8uG5/ZDki9KlkQUrC/uqGSFx7a+LmuomZ+DhNUpmWza3MEjI3zVm/z9nBa5+8wy5EljBmKy+LdZzGFM6ZWzRRIGREXF2z14QzINotevsDP/orWaNv3SKY0iWR6yaQwjR5btd8g5l3f9XOirTaYvnoS4rzmDykO2VLX+GNnHvxGx646nXI7+qoX2ha9NabY7zJRNxQUf8hbPDYz0eiFMTR5CQY7JUtWhx1o7XWSqm07yCl1E2E3TsMHDiwpc1ot/h9luum5SLTkObEGWU1jXQtzEn7OG6um0QWXCp4o6dlS1HorcO6CaLVzEM1jXQrzHF03oUMzUV/mkf/bgXZt+jNwzTEmTXKjttp/elrn8XkkLFrrv0BEn1e7KNx3/98L19/ZgnL7j2rWddBouei/XrI9Xkcwhd5sNq+m9aaq/+6kP3m90rXFWhZ9F6PgkObOH3NT9mrC3l3zGN8o/foSL0fn380xfl+enbK5fPdlZH7rjmDUO3b+LxOH32iSUWisfcvtBXNbcFeyyVj/rcSL+8EBtjq9TfLYtBaP6m1nqC1ntCzZ9u/2rQWfk9mhBLSSwk7b/0Bxj8wizkJcmLHw36DWhd7S0Uypc5YlzJLyNysS61h/d4qjntgFi8s2uYYgBIIaVbsqGDmyt0JXQmtgWVx/2LmF0nrGlpHJgi3cEsUtnDzIX72Wjgewi4y0T9Lte1h9+T/wong1u51vvmkSqLfPGjmn+lRlMPNpw5x3c5u0Rsa5m86GAlrTkXob502NHL9WW/G3SiHZ7+MoXxcE/gxVYWDHNv0KMrl5xeOpjDHct2kf91ax7S/BXiU03UT3T9kZ3TfYsfnbOa0iUdzhf514Hpz+XrgNVv5dWb0zWSgwubiOSKxOmNTFelTH5rDr95yFwi3Dsx4LNka7lZZtjV22H0yXDtjM2wNu4YVuln0SdwuWw7WAjDr871xXTfWbjPhPkuFVE7V7S8so/TumZTVNpLnT34bVtQFeHr+VipqAw6RjE5f7OgDMlc112OVSCSDIU2/LvksuecsxkRllnTr/L7vDWfys9oko0I75/u5a/rIpjdBpTjJs5onGu6Gyt28evTv2KZ7xxgQFldOHMg1kwfy7dOPMjdvjknftOj1RLlu4rxdjxvQhZnfOcW5m3Yw728q4ZX/AuYDI5RSO5RSNwIPAmcppdYDZ5qfAd4CNgEbgL8Ct7ZKq9shWms+2XggpsPU57WiVlL7lbcdquXJj9xT8qZj0UfHD6eDszPWPHYL30iiXQx2oX9s7gbeXLkrpo6V5wTcxVOjIyJZHww55l11f2g0u/lpkYoV+dryXUB4nIXXo/jqpNTcl9vLahNa9Jkc/ZtQ6A0jYsTkRr2RuLnKnokaEJXMx20d27p8x+96gedzfhX+cP0b7CgaC8RGc1nk+b384uKxdM73J6znhps/3xtj0ceZ49flMO3AoE8p6uYqrXUfrbVfa91faz1Da31Qa32G1voorfWZWutDZl2ttb5Naz1Uaz1Wa72k9b9C++DFxdu5+q8LeXOl8wXGl8B1k24UTaKQtnj7bo4h47C2LR99BvJvxzvGQ++s5VvPL4s5RmPIiNRzO1daN/lu6wOG43XazXpvaWrZVEnHzeXzevAoxVG94g9AsrP9kFPoo81Eh0XfZAxzsLqBfy6IHX1qce2Mhdz23KeOskS/+Vur9kSMAH+cjvZ03X3nH9MUoGc3VMaqTZyy5RE+DB3DNwr/AANOiAh4sr6HkSWduPHkwfz56tSH9DS5bpqIjrpJ5Lqx8/Dlx2Z1ysB4HLEpECx0htKF7jJHKK6PSq1gZU50c31EW/nJbox0bhyrZjrRBpHjOFw34f/pPGRc9xnVdjdrMVqIG4JGQovevt+GYMjhC3W16LMk9OlEZNQHQng9imMGdEmp/t7KekeKhOiv9MEaW5+Mbd13XljGxxsOcuKQbgzr1QkIn4+lW8uYOLhbJBncn237Sna+rPl/o1MIB5sp9Pb2GlrD3s941Pt7TvUuozanO9+r+ybdvIUAfG3KYIpy/VxxwoA4Owvj8SjuvWBUeu0wUVGum+XbKyOf41r0UZ+H9Cxq1j2Yadq+O7iN+ccnWxj103davB+rQ60uaiCIFRbm5l9vjLLykw0iaY7rxqPCHXTpvD24xdG31L8dLexBm0BbRJ+PhoAR2c7Q2tHRCGFdsLYJW/QGp43oSWGO1/XBlK3wylStPQgLvUcpjhvYlZe+eVLS+nUBw2HRR18T9sRudvZXNcTUf/zDjVz+xHw+2XjAdZtkD3fLUPFFTQriNmtWKoRz22j6q/38jCfh8ZOZzGpeDp3Cp2f9mwunjOOJaycA4Te5qycNTJrOoDk0hVc6M2bW2fqAHnx7Dalgdxm1pWV/xFv0970RTvUaMnSLLpoCs5e/LiqkTkXCE10s+mC00McKRLwcJ8mw6u6qqGfMz97lpxeM4v+dPDilbTfub3orSWTRv7N6DzvKavn6KUNi1kUT7bmyHiYVtslALDGyeOidNeypDL8pzV27nzE/e9e5E910DsOCCbk+L36fxzX8LVMTWSejLo30s/UBI3LdjU/Bqq8LhBydsYmiV9aZ4wyCIW178Ddd4xvNt894s3elKtTRceLWtZLO+V77i+m88tLzXJzzKBM9awnihUm3cOa849gfLOCZ4r78dEJ2ovNs/b8RivOT5693w+tRtlTTLWxYCzjiLXqLdC3Wt1ftdgii1SlY1+jcj3WzOLIpmjdn9E3qJvSO0Z7p+OjN/zvKwlEp75gZ/JZtK6P07pmO3PDR3PJsk69WJbDob3l2aUohhBC/M7aitknoo+cH+M/SHRGXQjys87qjrI6ahhB5fg/dC3NiHhpubWgt6tMY71Bnum4gNgTVtX5j0BHxkSie20pnHDCa3ozsYmMdL955SfWeOLqPM5ywPhDib/M2p5zIb6pnBbkv3cAVX3ybIWo37/S4gcAt82H6r9kfDGea7NaMcQAtxS70BXFmUplY6kx5HO0GTqcTuDU54i16CzfrZXdFHcGQdk1r+k2z48qaCcfS7LqA8+JuEnrbzRky2FlexyHb6EWttavrJl5GxmRYN3Z0CgEromfJ1kNccEzy9LbWddrSAV/RnbFBF4u+rDa9iBGNpjHU9HDcWV7HKf4elHTOi7wJJGpDa1GfwkApi0M1jXTKS/02rAuECASbvkcgZMRxy2l6UsGJns8ZuHIBP6xbSZec3fR7oyfk+qG+nK9VFzLGV0BR+TciW63cUc7w3p3I83tTjlLyehS3TB3K4x9uBOCpeZv4eMNBLhrnvL58BBms9pBLI0PVLgaqfZzh/ZRxnk2wvYSlvS7lG9vO4pvHTCS/ZKhj22wKvZtP3RPHJPb7nHWjt7Rb9G2JCL2J22vmib/+AGgScws3n6Yl5NFWuSWw9vKGgMHU386Nquf+ym9/QKT11mF+HSsO2fp21tD4bgWp3TjWRX/Pq6u5ZvKg1I8fRbzOWLvQl6c5p2tD0Ih5g/J5Fb2L85i/0TkxhkdlrzM2HaEH2GqOBUiFukbDsf/GkBG5drtSyaXeeUzzLGekZzs9lZlm4DPopLqzmZ74qneD7gz53eh+aDNXereT9/F7LMstYoFxNA8+tpqBx57GrWeOYcX28shxiqmmWNWyQ/ckRs52LeeEPa9wds6bDFZ72LW3lLX+LvT9/CA359TSXVVQr3PooqrprJzfdbVRyqPGl/n2HY/z1tsbKNu22TXrZ5tY9Nj96+5qnetLPGeiz+PJ6tywcdvR1g1oL6QzWcLVf13o+BwMGfzs9fCAkGiht8TF/hob3elo1XNz3djdNc2KulFOi956i0g02bKdeJZMukT3L1izFdkfbulO7bhyRwUrd6xylJXVBBjUvYB9Ua6bsf27sN/Fys8EVfUBdlfUM7BbAev3VlMXCHHt5EEJwxmbS10gSE1D+Jz5CdKnfDn8bzZ/9b/JWd7wW+YGoy8fGsfyuTGIpcZRfP2yC/jdB9vYcrCWf198IhPNGZb+/OpqXlvwGU8du4GKz2dzlmcp53oXs2/NEyz6bCSXe0dxnFrPEM9uJnrWAjAvNJo3jJN4KzQJhYYPfwtzfskZaNYwgA+M8YwO7OAUz3a26t4c0MWsNkrJUQGC2sdyYyjluoitujd7dVf20ZVOeT6+7cuxZf2MFcbWnBs3mqZ5epuu2aNLOrnWje6fiO7n83rbQ8yNCH2EoKExDE3Q0AlzUxxySRY17CdvR5btgv6799ZG3AU1NreMm1/V0E6hv/2FZeT7vdx59nBHG1PF8rta192hmkZeWrojMotTqm8HqeRUT4Xoh9SizYdYtPkQP7WFvrnNMJUu+6rqGTegi+N4OV4PI3oXsbu8LnaDQD00VkPFDvB4w59DjTBwcvhzClz3t0Us21bOt04bxp/mbAAgPyf+tueNLeGtVXsin08e1iOl4xRTTb/KldBQwXm+OXzJO5+CXQ2wC0Z7ujEjeC4vh07hM13q2O611WVNo4NtRobXo6ikiBX9ruKXK8ZRwkGmeldybc5cTlIruMD/MQCbjBIeD36JHqqCqZ4VnOz9Kz/yPY+fIMxpgJEX8GThTfzq4ypAQZpjtqyItWLThWV3ZU0a3I2Fm6OT57Yu1gAwu4vsmsmD6F2cx03/XOqo6/clEXqlYkYvtwUi9CYhQ/OD/67kpU93xLhq5q7dx7QRvYBwPnQ70f5Re/rSRz/YwDdOCUe6OFw3LuF3IUM7/PvWyMnbzzzKUachGGLDvmpG9+0csw9Hu8z/lpW09WAt3/tPU2rYeG8HzXFvGIaO25H4r0XbmLf+AKOi8n9Y/G99U6bPymakcLZ48abJXPHkAsprAxTnOy/rTjkwtvoTTg2+DU/cD9X7wOMHIwBVcTJ0dBsKQ0+DE74OPUYkfLVZti3s4thre2PIS2AsjOnX2SH0N0+NH7XUlUou837Emd5POV6tw7c3LNQ13lxeC53EXGMcfY85g78vr4y7j/e/2Ev/rvlA+G1y3d4q/rVoW+Qhbl2be+jOi6HTmO09h/K6Ws70fMpmXcJa3TRqV2Ew1bOCq7xzqKKAy274LgyeSvXsjUDzpue0rPVbTxtGl4IcLj2uf2Tdc1+flLVoKQur47Xedp8qpTjj6N4xdaMt+mjDyD7tYbbSjLshQm8SCBm89OkO13U3/H0xv7pkLJce189xM0OslV1e4zRnnjPzatstfbfBFqf931xGuLwe2l03QUNz3xuf8/zCbXxy9+n07ZIf9/tYAh/vJnl41jr++P56/hsVux3tX7ZbKPEEvTFkkBfH+v3Ry2HXSm4c4Zuzdr9rebocO6AL104exJUTB7D1YC05BChRhxinNvINZjF2y1oOUUylZzyBHoPoXpQbFvuiXtCpDzNWNbJh2w4qQ7mc3DvAVZ0/g8VPhf9yi6H0ZAL9JuIfeAKUHAP+AvA6b5/OthC8vAQWfTR5fi+UbYXFT7Fq7FZ+93lneqlyRnu2MNWzEoCarkczM3QlyxnB3nofH1X2phozSCCByFvsKAu/zXy6tYwb/r7Yse73769zfG4MGgTx8Y4xMWY/Gg9zjfHMNcYDcNnQ04H0hvkX5focYyKsayPP740JAfZ5PSRxg2cc623M7V5Ydu9ZjH9gVqQs+u0/+v7olOdz9EO1FSL0JnZL1m207I9fWcW6vVX06ezMBx/tV4+e99Ran8x1s6+qIcavDM7OWMPQkQ6yA9UNvLlyF4N7FFHTEESjuWR8kyVkGQ/xwu8sKzSa6Ivbfl7iCXpDwHD4ULXWrN5Zydj+TW8dyUaL+r2qRZE9uV7FAxeNhi3zKH3/u6zLWx9ZV6ULeGPAXfx0+3jKNoaPEZ2694FXZwLhSI+Zu+Gq2x+EQ5tg2wLYvojaNe9TsPatpgP68uHU78HEmyNFh2x9DHk2dfISor/azwS1jjzVyMnrX8fw1fGxMYalxnCGf/gt2PwWoOkE/Nx8XuyiB48HL+Aj4xiev/2HvPPsUtbureJQsJHqdP0jJo98sCFpnea8WaXj4fv5haN5bM4GNpkja48b2DXJFtmlSehjr9loYY82YKLGjpHn91IpQt9+sFu+gZAmxxd75e6pqI+5oFNNIlXb0CSg6cRZ23OXBA0dubDqAwa/ipph6ZLx/Vmy5ZDjYkwlAuS15Ts5eVgPuhflUh8lyDFC79Ip1hAKAf7I8f6zdAf3vrqaZ/5fk0WYbLRojtdDIJRatMovLxzBtODHLPnoTWioJogX9djP4UDYMvUX9efhwGXUkMd8YxQNXYczrXc/Atu2A+HzeebDH7L03rMIhAzH7EUOug0J/427mof0Z7z+yUp+NSnA9J4HYcvH8MEvYPEMZvhLOKA703l7d3Z5jqZB+5m84R0e9a8jj0YmetbSWdVEdhvYnc9obz3f8r1GhS6g8+ZamHI7HP81KOrFW/MW8533KijKz6M82PT7l/Yo5J3P9rTJwJvvnD4s5iFx0tDukeUTouLJE+H3Kt78zsks31ZOrt/LmH7ubr22wuozcLt3ooXe74111cTQDnpjRehNnNPQGeS4jCULhAx2ljk79KKH5cfDXi9ZqgN7AiW7q+j+Nz6LWL1bD9bEbFdRG+AycwrD604Mh0Ime6hs2l/N7S8s58yje/HU9SfEXNxLbWmO7SN5V9tmCGoIGLyzeje3PPspSsGl5pvFJtuAsve/iJ8Xf873p3HpYx878snbyaWREzxrGaG2c4xnExe8vwSv0UhnCvB7TCu60xToexyUjGV36WU88khTp9lQrx+vRzk6oA/WNDJz5W5ue/5TZn9vaqJTBIRHRh6imM8LhzH95BEw5Q7YtoD6939Jn4ptjPFspndVOdPNlwRjm59Oqgt15LBIjWVBYCirjCHUkcu5Z09nxruLudg7j5FqO6dfcBXdJ18dOVZN8VCCrIx5qxxZ0iki8pdP6M/5x/Tl8bkbHXOrthaXnzCAf3yyJWLtXz6hP7+8ZGxk/ZRhPXj79lM494/JpzD0eTwU5Pg4KcVO6GxTEMd1A7EDoGJcNy6vNu0h7qZDC33I0Jz+u7n8cPpIzhvbsqlp7SIQCGmedQmNawwZ7D7k9NFXp/iaa3ddJHs42B86dqG3v1Kv3llBHg2MUNspVOE69z2xj+NUOUWqjiGVeznPs5mKxtEoCtEoiqijmnzsJsZOMxLlo/UHmPLgB/zfV46N2257WOhVf13gKLdG02rd1B+RqiumX5d8h1XdhSqmelZQosoYpPZyjncx3VW4E7xSF7Br8GUMmHgR18wqZPmOSjwYbLr+S5HtO0VF7/g8npjsgwDvfhbuEF21I3ZaO8vSD4bC6Xitkc8VdQEMQ7OvqoHlVYN50fNT5jSG+xnGd6nhKL2Vhy4+Gk+fYznlwbB/vTjPR2Wo6bf7yaBuPERnZoTCnf5Lx5zpbK9pJUaLxtCeTRkuLxrXjynDevDO6j1ZEXqPUo6Q3Hy/N+ZNqGuSsRlfnTSQ5xZuY0RJapk624qIRe/ibox++EbH0VsW/ft3nkpRrt/cJryuLWNvOrTQV9cH2Xqwlh+9vKrFQu9MNWBwz6urY+o0BIwYS7o5k32n4wPdW9mAwuAEtZYSVUap2sMkzxf0+bSMH+XuJ0/ZXEcVQK65vBFuyAFqoCHXT65Zr0bnUkcuc0LjCODj9X/MZawaiD8UpLYijw/XNp3H4zqV06lmK6PVZvyE+PfMIPk9S7lu2hiHjRI9CYP10KxJ8uZSlOsjP8dLjs9DkTfIAQzO9izhl/4ZEWEv00Ws9QzlnoZpzDdGUU4Rf584kQEjevF4ST1n/G4uXQqcndLRI02VCoe5RT94Xl8RjmyyOozt1DQEWbWzgmtnLOLNb58ccb1tOVjL5F/PjvSn2IfAr6stprbrJDj6VABumVrDa8t3xhgDk4d0d3TqRbvDLIGPNg4Hdm8aoT3QHK3do8hdXIf0LGTT/ti3vpYwbkCXSHZMN9FKlCvq7187gWnDe/L9s0c0a1rDbGJ1pKcyXWC8zlgrQyi0C89NxxZ6S1CShS19vquS+mAoYadP0NCuy3Z2lNXGiEV052sqJPPr96SM8Z4NXOz9mPFL93Fb7i7yVdhKNbTiCz2QvUYXPtRjmW+MokIXka/qGdkjh88PBKnReZwyrDuLNu7haLWN/mo/DeRQq3PpqqoY5dnKhd5PMPBwtfrAcewDy/pwQ249RdRRFKgHyxWhFZ51L8E6qPh0AI+qEmZ6x+FBE9gYoieV1JNLFQWR81ceJ6VBDgFGqm3cOsrH9N5V8NyTzKqfhTcv/HtuzxvBjRVfZaPuSxX59CjK5YDRZKX7zVDHks55fHb/9Jj9R1uaNY3BhHlkohPRQfit680V4dDLFTvKI29h9onfARpsbzk1jSHHbFF3nzuSu88dyU9eWRWJvrKwi1200FupraMt+uK8pqgeK1zSbcRont/DdZMH8XMzYV8q9OqU6xoMYNEYNHj0qvH8+JVVvLZ8l2uCvXhC/9zXJzHFdNO0d5EHYqZ2TES0Kyd6bmRo5uxWGaZDC73lVkj2SnTeI2G/oRUf7/ZgCDk6Y92f5LtcEoE1Z0YfN3dPUY5iRGANl3k/4nLvXLxKs093YVXdYEZ7KvhN4EqWGMPZpntRicurr4atRiGbjbAV5wl0ZbHRg08Y49oGhYEHzXFqPZ1ULTkEGaJ2M8m3nUajHh8hqvpN5bmtnVmn++HD4GLvPHIJ8KVuBxhZv4qpfjNMb/ZTLDaDkbYavdhyYDyjvF3otH04p3gCDFJ72a57cZJnNeM8Gxmv1pOjQvAF8IWCbkOYmXMuu2shWNiX237wa5b/5D3bOXaer3SzjFbUBlxvwETUNISoagj/ttHhgHbsaQLAfQTn/ReN4a5zRnLs/e85yvt3zWdHWV3sIBvzWeE2cO/0kb3oUZQTEQ+7KA3olo9XKd6+/VReX+GertgiunO1pHNeQqHvVZxLQY6PCYO68tryXa4dwtHn+J07TmHd3uqIyHcU4nbOR/HrS8fGlLldm9Elr39rChf+6ePmNK3ZdGihj7xapen8chsU5Ez9qhnSozAS/hWNfZ1dtI8d0CXmxnfDLlyneFZymfcjpvm+oLOnjKD28EzobGYZx7PYGEkgwU9UnOdzuIHsHcWJbloIx0OHgMV6pOP8PWYb8/L94cNZuKUpxvqvoQsAaBw2hI/W7CawfwMN+PnhCV6WL11ILgHO8i5lVPUCpvorYD9802bAhbRim+7FK6FTmGOMY8qkSVx77lTIKeSpP81jZWUFI/M7cZvXmRK2IWjw7I2TeHPlLs4ZXcKkwckjPHweFXmzqKwPpp2d9EB1Q2RQk6E11fVBhvcuYsvB2phX+gcvHcvdpvvHbUSs16PMXOtO3vjWyWw9FJvnxmta9NGDcQD+dsMJjs+5tjeI+y8cw2kjwwP7rAdOvy757Cyvi+mjiJ7+r1enPMK+PyenDu/piJ6y/EmurpuoCJSRJcWMLGlfETWpcs3kgZw1qiRhnasmDoykubDCg12nEowqO6Z/apPMZJIOLfSpWvQW1Q1BKuoCHKyOFcFoH/2AbgVxhf6p6yfwy5lfMHvNPodof//s4Vw7Y1Hc4x/TtxNVu9fRrbySSz3Lubf3fLqWreCALmZD4fGMOeOrBAdN5b7fhC3lwhwvgQS5zbsV5jiE3t5Zmk6irHj07JTrWv7kR5so7V7AFt0PgGX+wfwtFA61eyx0EQAF1FOq9tBZ1bBPd6EzNeynM9t10+jCEQXDICc8Y5BloVpJ1yYP6cbiLWWR3+Xko3pw8lGpW4YFOd7IuRlZ0ikyaMUSvmS8YfrvAaobQlQ3BCnM9ZHv98YI/clH9eCqiQP416LtruIcj66FOa6uDGsX0RN6uNGvS5Pf3u1N1Oqv6Ncln222h0pp90JHvV7F4d/aekDm+jw0BGMzY1otSsWi78j84uJYa92NQ9Xh67VLQTg1dqJz0JZnp0MLvXXDRV+Mr6/YxbPzt/LvW050lF/854/ZsK+a6aNjn9TRcfSJ4r4H9yjk22ccxew1+6JG+DVZSSNLOrFmTxU5BLjI+zFD1G7OrVhJae5W2ALkQE1oCEtG3Mm1K8Zw6ZDhHD9uLHZp/e5ZwxPme+9ckAMZEPS4+08w2cKWg7X0KMrhQHUj+10enLXk8bkupW9xnqvLC5y/2xe7wqM7rTzyL9x0Iqt3VnDBo/NI01MDQGFu+G3nlqlD+cYpg7n/zbC/+s6zhjtSQUD498zxelhrS2/xwuLtTd+lIcj+qgYGdi9gu4sF3rs4j+G9w51vRbnxb6nZ35uaUn5yy6JPxYUwcXC3iGvJfg1bA/VG9SlmaK+iSB6emSt3U1Kcx3ljnfeA1e7uRTnsrWygIMdrCr3zeE06lrqP/nDmgHntdy3wh4Xe1XUT/y0oW3RooW8IBPmu77/MJjyMf8uBGvp1zec7/1oGhB8Edj/nBnNGnQHdYlMHOKdnM+L2uN9x5lEopSIDl5ZtC8eZX3/iILruX8zN3jcYrHZzUuV6CnKr6EYVHqVp0D4O6R78PHAdW3VvDulO3HPJV1m0pZy6FWvplBcrqm4pDkq7F7C7op6GoJEwn0om6JyfuOOsW2FY6O3WbzTdi3LjCn0f2/dzM4SsB01zUtR2zvezu6KeCYO60r0oN9Ix3K0oJ8Yt9/btp9AQMGJ86BY1jSH2VdVzwuCurm4/v9cT6dBN1FZ7eGQiLKswVV/xtBE9eXPlbodF38U8d6P6FkdmADt5WA9mrtzNlGE9UEoxaXA3th2q5ZapQ9lVEX7LsXz++X4vZQRiOl0jouVm0dtEblSfjumySZcmoQ//7q6d/u3g+dehhT5UV8Htvpe5SULkkNYAAA+SSURBVL9JWc31TPu/uVw9qSkBU2V9gB5Fse4Ht3TAv3tvbWT5wj99HDdsrUu+H4wQXbe+wz2+l6nbk8uXfTVcsWULucvW8yM/BLSXxWo8m0MjKaeQhcbRfGQcw/DendjSUEujYeDzKMYP7MYHa8O57d0mn+jl4jqZ+4PTeHbBVu55dbVrZ92AbvlsP5TcNZEIy73RxcWvbCcVAY73VvC3GyYwbXivyGfLDfdlW0KrPp3z+NKxfSOJ4dJhSM9C1uypinSoWrnuO+f7efiKccyYtznygMrzexO6XMpqGimrDdC7U17cpG+XTxjA0i1l3DJtqOv6dLDyp6XqBrIeCPa8SNPHlMScYysiyOqvePHmpjfeH78S7mOwrikrxDBG6BNMi2fXuLduPyWltnd0fjh9JHWBEJMGd2fh5kPt1n3V4YUeIF81cuPz4QE7c9Y0jcD8+8ebmTi4e8x2bnHs6/Y6M+8dqLYPutEMUbsZqzZx0qZZsPgDSsq3co3Xj58gQXwYBRPZP/o6zni/hAb8dM8rZleN05Id1quI/VUNNNYafOnYvvi8Hs4e1Zu/zN3IWaNiM+NZIml19PQ18+xYr9l2C8ryqZ4+ohez1+xj+ugSnpq3Oe65i4dHwcu3nsSry3YyMirJ2uPXHOeYZjCRa8figItbB+D0kbHfF3CkZfZ5PTx61fhUmh3DiUO689aqpnQBN5w0iO++WM6wXkUU5/l59Krx7C6vo8h8wHo8Cq9H8Y1ThkRmSrJ4/4u9QNiPHU/oexTlMiOqo7S5WG2Onr0oHl+Z0J9Xlu10pCFQSsWcYyts0y182OprsoTeGh0a47qx2ujiiGgPYYTZYPV950TcjqU9CvnH1yby94/D91p7dV91aKE36pqiBEZveZr9ahzlwcF4VHjGpj/P2cif52yM2e5QjVN8cmmkgRx8BOmrDpJPA7Xkcrn3Q871LGKQ2otfhd8CQpv9MGAiNdN+ztgX/BgoPGg+uuwMDAMq358DQCDqZnr2xkmMG9iFy/7ySdg6LA6L9viBXWPSIlv+/QJT0If2LOKXl4xlkDlgxuqksyfOumhcX84d24eJpd2476JwSGWqQn9s/86s2FHB/ReN5roTSwG4eWrYMv3alFLG9uvsSB1rkSxRGaQ/IXJxGtPqJeKayYMo6ZzP6WYUyiXj+zuSvgExmTs3/uo8AIfQ9yjKYbfpeupVnBdJ03DZ8f3579IdCX3yzcXqVE/VdXPS0B4x15AbVv+A2yQ7xw/swhsrdnHP+aN4ZPZ6rjhhALe/sDzm97Ni++NlDr7w2L5ceGzyKSo7Mm6/uWUAtNdnXYcWel3fJPQ/8T/PT3ieT0LHcov3TiqDdmszMt8SAAerG7nAM58LvZ9Qog5xjGczB3QxnaiLjCAFMFAsN4byTOhstuuezDdGcdtXzuXC40opBM5etZR3PtuDgaKHzQ8M8K3ThkVmnQIiESPWjRKdBdPOK7dOoT4QokuBn7umj+Dicf0c/nrrhjW05qEvH8NdL63E0HDaiF6u+3vsq8dx63Ofuq4D+P45IzjlqJ6u6372pdFxt3PrmOyU54tYh/ecfzTTRvTkzIc/iqx/6MvHMKZf/Fz6mRJOpZTrW1Iq/Pnq47jNfEO894JR3P7CcgB6d2r6zW6ZOpT/Lt3BCaWZz7xo5RTyZWp6LxMrhYGbRX/9SaWcM6aEPp3zmTykO4vMyT5iLPcErhuAR5r5BtbRsYQ+YWe77ZwN61Xk6pptLTq00KuGcKTGnNCxfGgcy5XeOZzkWcFK3/Xs9nZjp+5Brc6lnzpAsaphiTGCQ7qYTfv7cLN/Jr1UGRW6gAXG0WwxeuPFYKvuzWbdh66qCk/pFBbXlvDF7vBxThzSndNG9Ysc/y/XHMfgH4VT1+b5veT4mnz/159UyvUnlVJ690xHmy3/6JRhsS4li/wcbyQe+9Zpw2LWW1EZhm7KD2/EM7Eg8ibgRnGeL67IuzG8dxHr9lZz6fh+nD26d8SVY02d9+hV4yP5zr9+ypCYiKjLTxjgut8TSruyeEtZu3j9P/+YPhTmnoBHKceEKVYIIsDQnoU88/8mctygVhB60+LO8amM9LlYWG+CbmMKlFL06dxkTFhZGaMHgCVy3RzJWLefW2es2yX9/p3JE+llkg4t9DSEw+HuD17HZt2Hf4Sm8yX/Un7i/Qf7dTH5NJCrGlFotuneTPGspliFb5oyivlyw89YqkfE3f2g8gKevfF4Tnko7I75102THeuVUozqU0zfLmFLL9EUhBZ/uGI8n2w84MiFkS5WJ+iAbgWRWPd+XeNPQtIpN74v/YYp6XV0vnrbFOoDRkxH7AMXj+GWaUPpFxUpZBfuZfeeFXe//7xxkmNylrZmmsvbUbeCHF765kls2l+NUopTh6f+gEwHq+9jcI9C/u8rx8bkEmouo81ImCvjPGztjBvQhe+dNZwrJjrrRn5P0XkHU4f35DfvrOEcl9Dt9kCHFnqPadFX6SaLdWbweBblncjeqnjzj2p6UU5x1x5sqE+c/3xiaTcGdCuI+MzdsEcXpBIlMapvcdxp9VLl+EFdeeq6CZx8VA9yfR6evPb4yIhIO29++2SKcn2RDsdonrj2+IgPO1UKcnzYkxTO/M7JkZC8aJGPJlGekzy/N6sTQKfDH68cx8LNh/B4FMcP6srxrWDF25kyrHvkN/V7PSRJCpkyvYrzUvLlQ1jQv33GUTHlljETPbL2SGdU3+KUz21b0LGFvjEsvlU0CYyh4YBtOr9bpw2lW2GObeCRYh9dKfLlAc6Rr9GC/otLwp2aL996kmtIZjTW6+4dtnleV/z0bFQrhLufafM/nx3HirB84fHGBGTC+kg2d+3hwEXj+nHRuH7JK2YIpVTc37StOW9MCetOG8Y3To0/z63gxPLbD+vddumZW3fETStz/NU/5y+T59CA0+QJGZq7po/glVtP4q7pI12tXfuAjuvNSTosN0i+38vc70+LjHQtyPG5xuNHo5Riy4Pnc8eZTSGCnQv8jqyDbYHdpXTO6PADojSB314Q4uHzevj+OSNSCq0VwhTk+PjnjRP5e4bCb5tDh7bo/X4/Qwb0BXbz7dOHceLQ7tzwt8WcNrInXz95SETghvYs4uapQ3jiw02RbS88ti8/OCfsn+9dnEee38ttpw/jqf9t5quTBkbCHw83HrlqPJV1wUicdGtw7IAujqyeD1461nXic0E4Ukgn4KE1UMlyuTdrp0pNB/4IeIGntNYPJqo/YcIEvWTJkmYdS2vNW6v2cNao3uT4PIQMHXfQwuwv9nLj00tQCjb96rx2EeGRLUrvnklJcR4LfnxGWzdFEIQMoZRaqrWekLRepoVeKeUF1gFnATuAxcBVWuu4syC0ROjTIRgy+O17a7n51KHNyp/Skdlf1UCe3+OaU0cQhI5JqkLfGq6bicAGrfUmsyEvABcBqU9300r4vB5+dO7Rbd2MNiFeymFBEA5/WqMzth+w3fZ5h1kmCIIgtAFtFnWjlLpJKbVEKbVk//79yTcQBEEQmkVrCP1OwD6crr9Z5kBr/aTWeoLWekLPnm3bIy0IgnA40xpCvxg4Sik1WCmVA1wJvN4KxxEEQRBSIOOdsVrroFLqW8C7hMMr/6a1/izJZoIgCEIr0SoDprTWbwFvtca+BUEQhPTo0CkQBEEQhOSI0AuCIBzmtEoKhLQbodR+YGszN+8BHMhgc1qTjtJWaWfm6Sht7SjthI7T1tZs5yCtddKwxXYh9C1BKbUklSHA7YGO0lZpZ+bpKG3tKO2EjtPW9tBOcd0IgiAc5ojQC4IgHOYcDkL/ZFs3IA06SlulnZmno7S1o7QTOk5b27ydHd5HLwiCICTmcLDoBUEQhAR0aKFXSk1XSq1VSm1QSt3dxm35m1Jqn1Jqta2sm1JqllJqvfm/q1mulFKPmO1eqZQ6LovtHKCUmqOU+lwp9ZlS6vZ23NY8pdQipdQKs633meWDlVILzTa9aOZUQimVa37eYK4vzVZbzeN7lVLLlFJvtvN2blFKrVJKLVdKLTHL2uPv30Up9V+l1Bql1BdKqRPbWzuVUiPM82j9VSql7mhv7URr3SH/COfR2QgMAXKAFcCoNmzPqcBxwGpb2UPA3eby3cBvzOXzgLcBBUwGFmaxnX2A48zlToRnAxvVTtuqgCJz2Q8sNNvwb+BKs/xx4Jvm8q3A4+bylcCLWb4G7gSeB940P7fXdm4BekSVtcff/2ng6+ZyDtClPbbT1l4vsAcY1N7amdUTkeGTeiLwru3zj4AftXGbSqOEfi3Qx1zuA6w1l58gPL1iTL02aPNrhKd9bNdtBQqAT4FJhAef+KKvA8KJ9E40l31mPZWl9vUHZgOnA2+aN3K7a6d5TDehb1e/P9AZ2Bx9XtpbO6PadjbwcXtsZ0d23XSEmax6a613m8t7gN7mcrtou+kyGE/YUm6XbTXdIcuBfcAswm9x5VrroEt7Im0111cA3bPU1D8AdwGG+bl7O20ngAbeU0otVUrdZJa1t99/MLAf+LvpDntKKVXYDttp50rgX+Zyu2pnRxb6DoUOP77bTYiTUqoIeAm4Q2tdaV/XntqqtQ5prccRtpgnAiPbuEkxKKUuAPZprZe2dVtS5GSt9XHAucBtSqlT7Svbye/vI+wK/YvWejxQQ9gFEqGdtBMAs//lQuA/0evaQzs7stCnNJNVG7NXKdUHwPy/zyxv07YrpfyERf45rfXL7bmtFlrrcmAOYRdIF6WUlWLb3p5IW831nYGDWWjeFOBCpdQW4AXC7ps/tsN2AqC13mn+3we8wv9v735dGorCMI5/36QyxB9gM8iKTQwGg0Ewubwiggb/ChH8E2xGs8UgZl1XxJ/TBWdSUKPZ8BrOe9mKYNl2dnk+cOHes4Xncu7e3XPOHUtfoLn1/zvw7u6XcXxCKvy55SysAzfu/hXHWeUc5kI/DP9kdQZsx/42aT68aN+KFfhl4LtrmNdTZmbAEdBy94PMs86Y2WTsj5HWElqkgl//I2txDnWgEXdTPeXuu+4+6+5zpOuw4e6bueUEMLOKmY0X+6R55SaZ9b+7fwJvZjYfTWvAc245u2zQmbYp8uSTs5+LFT1Y/KiRnhp5BfYGnOUY+AB+SHcjO6R51wvgBTgHpuO9BhxG7kdgqY85V0jDyAfgLrZaplkXgNvI2gT2o70KXAFt0lB5JNpH47gdr1cHcB2s0nnqJruckek+tqfic5Np/y8C19H/p8BUpjkrpBHZRFdbVjn1y1gRkZIb5qkbERH5BxV6EZGSU6EXESk5FXoRkZJToRcRKTkVehGRklOhFxEpORV6EZGS+wU46aM82Gdq9wAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(score_log)\n",
    "plt.plot(average_score_log)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for episode in range(3):\n",
    "    state = env.reset()\n",
    "    score = 0\n",
    "    input()\n",
    "\n",
    "    for t in range(max_t):\n",
    "        action = actor.act(state, add_noise=False)\n",
    "        next_state, reward, done, _ = env.step(action)\n",
    "        env.render()\n",
    "        \n",
    "        score += reward\n",
    "        \n",
    "        if done:\n",
    "            print(score)\n",
    "            break;\n",
    "\n",
    "env.close()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
